library(chemhelper) #for sim_*(), plot_pca(), plot_plsda(), get_VIP(), and get_loadings()
library(tidyverse)
library(ropls) #for opls(), which does pca and pls-da
library(cowplot) #for plot_grid()
theme_set(theme_gray())
library(iheatmapr) #for correlation heatmaps
library(here)
Read in datasets
I’ll read in the datasets from “Generating Datasets.Rmd” and pick individual datasets to make figures
null.list <- read_rds(here("data", "null.rds"))
control.list <- read_rds(here("data", "control.rds"))
needle.list <- read_rds(here("data", "needle.rds"))
set.seed(999)
null <- null.list[[sample(1:100, 1)]]
control <- control.list[[sample(1:100, 1)]]
needle <- needle.list[[sample(1:100, 1)]]
Null
The code chunk below creates a data frame with a grouping variable with two groups (a and b) with n=10 per group, 5 variables that don’t covary, and two groups of 10 variables that moderately covary.
# # myseed <- runif(1, 0, 1000)
# myseed <- 401.6368
#
# df <- data_frame(group = rep(c("a", "b"), each = 10))
# no_discr <- df %>%
# sim_covar(5, var = 1, cov = 0, name = "uncorr", seed = myseed) %>%
# sim_covar(10, var = 1, cov = 0.5, name = "corrA", seed = myseed + 1) %>%
# sim_covar(10, var = 1, cov = 0.5, name = "corrB", seed = myseed + 2)
This heatmap shows the correlation structure.
# no_discr %>%
null %>%
select(-group) %>%
cor() %>%
iheatmap(row_labels = TRUE, col_labels = TRUE, name = "cor")
null.cor <-
# no_discr %>%
null %>%
select(-group) %>%
cor() %>%
as_data_frame(rownames = "row") %>%
gather(-row, key = col, value = cor)
# cor1 <-
null.cor.p <-
ggplot(null.cor, aes(x = col, y = row, fill = cor)) +
geom_tile() +
scale_fill_gradient2("Correlation Coeficient") +
labs(x = NULL, y = NULL) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 90), legend.position = "top")
Then I perform PCA and PLS-DA on this dataset, forcing both models to have two axes (for plotting). The PCA score plot shows no separation and the first two axes explain more than half of the variation in the data. The PLS-DA score plot shows slight separation of the groups (even though there is none). It’s a pretty terrible model with a negative Q2 value and a low R2 value.
null.pca.p <- plot_pca(null.pca, null$group, annotate = "subtitle")
null.pls.p <- plot_plsda(null.pls, annotate = "subtitle")
plot_grid(null.cor.p,
null.pca.p + theme(legend.position = "NULL"),
null.pls.p + theme(legend.position = "NULL"), nrow = 1, ncol = 3)

Needle in a haystack
Keeping the total number of observations and variables the same, and using the same random seed, I create a second data set with 5 variables that do not covary within groups, but differ in their means between groups.
# discr <- df %>%
# sim_covar(10, var = 1, cov = 0.55, name = "corrA", seed = myseed + 1) %>%
# sim_covar(10, var = 1, cov = 0.55, name = "corrB", seed = myseed + 2) %>%
# sim_discr(5, var = 1, cov = 0, group_means = c(-1, 1), name = "discr", seed = myseed + 3)
This shows the correlation structure of the data. Because the discriminating variables have different means in the two groups, they are correlated about as strongly as the covarying variables.
needle %>%
# discr %>%
select(-group) %>%
cor() %>%
iheatmap(row_labels = TRUE, col_labels = TRUE, name = "cor")
needle.cor <-
needle %>%
# discr %>%
select(-group) %>%
cor() %>%
as_data_frame(rownames = "row") %>%
gather(-row, key = col, value = cor)
needle.cor.p <-
ggplot(needle.cor, aes(x = col, y = row, fill = cor)) +
geom_tile() +
scale_fill_gradient2("Correlation Coeficient") +
labs(x = NULL, y = NULL) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 90), legend.position = "top")
# needle.cor.p
The PCA score plot still shows no real separation because the discriminating variables do not contribute much to the total covariation in the data. The PLS-DA plot shows very strong separation, despite the differences between the groups being only due to 5 out of 25 variables. The PLS-DA model performs well, with high R2 and Q2 values
needle.pca.p <- plot_pca(needle.pca, needle$group, annotate = "subtitle")
needle.pls.p <- plot_plsda(needle.pls, annotate = "subtitle")
plot_grid(needle.cor.p,
needle.pca.p + theme(legend.position = "NULL"),
needle.pls.p + theme(legend.position = "NULL"),
nrow = 1, ncol = 3)

Control
control.cor <-
control %>%
select(-group) %>%
cor() %>%
as_data_frame(rownames = "row") %>%
gather(-row, key = col, value = cor)
control.cor.p <-
ggplot(control.cor, aes(x = col, y = row, fill = cor)) +
geom_tile() +
scale_fill_gradient2("Correlation Coeficient") +
labs(x = NULL, y = NULL) +
theme_minimal() +
theme(axis.text.x = element_text(angle = 90), legend.position = "top")
control.pca.p <- plot_pca(control.pca, control$group, annotate = "subtitle")
control.pls.p <- plot_plsda(control.pls, annotate = "subtitle")
plot_grid(control.cor.p,
control.pca.p + theme(legend.position = "NULL"),
control.pls.p + theme(legend.position = "NULL"),
nrow = 1, ncol = 3)

Save plots and tables
fig1_new <-
plot_grid(null.cor.p,
null.pca.p + theme(legend.position = "none") + labs(title = ""),
null.pls.p + theme(legend.position = "none") + labs(title = ""),
control.cor.p,
control.pca.p + theme(legend.position = "none") + labs(title = ""),
control.pls.p + theme(legend.position = "none") + labs(title = ""),
needle.cor.p,
needle.pca.p + theme(legend.position = "none") + labs(title = ""),
needle.pls.p + theme(legend.position = "none") + labs(title = ""),
ncol = 3, nrow = 3, labels = "AUTO")
save_plot(here("figs", "fig1_new.png"), fig1_new, ncol = 3, nrow = 3)
old, not run
VIP scores from the PLS-DA model identify all 5 discriminating variables as being important for separating the groups (VIP > 1). Using the VIP > 1 cutoff, it also spurriously identifies two of the other variables as being important. Discriminating variables are not strongly correlated with the first two axes of the PCA.
Red Herring
Large variation with weak discrimination plus small variation with large discrimination
LS0tCnRpdGxlOiAiUENBIHZzLiBQTFMgd2l0aCBubyBhbmQgZmV3IGRpc2NyaW1pbmF0aW5nIHZhcmlhYmxlcyIKYXV0aG9yOiAiRXJpYyBSLiBTY290dCIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpsaWJyYXJ5KGNoZW1oZWxwZXIpICNmb3Igc2ltXyooKSwgcGxvdF9wY2EoKSwgcGxvdF9wbHNkYSgpLCBnZXRfVklQKCksIGFuZCBnZXRfbG9hZGluZ3MoKQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShyb3BscykgI2ZvciBvcGxzKCksIHdoaWNoIGRvZXMgcGNhIGFuZCBwbHMtZGEKbGlicmFyeShjb3dwbG90KSAjZm9yIHBsb3RfZ3JpZCgpCnRoZW1lX3NldCh0aGVtZV9ncmF5KCkpCmxpYnJhcnkoaWhlYXRtYXByKSAjZm9yIGNvcnJlbGF0aW9uIGhlYXRtYXBzCmxpYnJhcnkoaGVyZSkKYGBgCgojIFJlYWQgaW4gZGF0YXNldHMKSSdsbCByZWFkIGluIHRoZSBkYXRhc2V0cyBmcm9tICJHZW5lcmF0aW5nIERhdGFzZXRzLlJtZCIgYW5kIHBpY2sgaW5kaXZpZHVhbCBkYXRhc2V0cyB0byBtYWtlIGZpZ3VyZXMKCmBgYHtyfQpudWxsLmxpc3QgPC0gcmVhZF9yZHMoaGVyZSgiZGF0YSIsICJudWxsLnJkcyIpKQpjb250cm9sLmxpc3QgPC0gcmVhZF9yZHMoaGVyZSgiZGF0YSIsICJjb250cm9sLnJkcyIpKQpuZWVkbGUubGlzdCA8LSByZWFkX3JkcyhoZXJlKCJkYXRhIiwgIm5lZWRsZS5yZHMiKSkKYGBgCgpgYGB7cn0Kc2V0LnNlZWQoOTk5KQpudWxsIDwtIG51bGwubGlzdFtbc2FtcGxlKDE6MTAwLCAxKV1dCmNvbnRyb2wgPC0gY29udHJvbC5saXN0W1tzYW1wbGUoMToxMDAsIDEpXV0KbmVlZGxlIDwtIG5lZWRsZS5saXN0W1tzYW1wbGUoMToxMDAsIDEpXV0KYGBgCgojIE51bGwKClRoZSBjb2RlIGNodW5rIGJlbG93IGNyZWF0ZXMgYSBkYXRhIGZyYW1lIHdpdGggYSBncm91cGluZyB2YXJpYWJsZSB3aXRoIHR3byBncm91cHMgKGEgYW5kIGIpIHdpdGggbj0xMCBwZXIgZ3JvdXAsIDUgdmFyaWFibGVzIHRoYXQgZG9uJ3QgY292YXJ5LCBhbmQgdHdvIGdyb3VwcyBvZiAxMCB2YXJpYWJsZXMgdGhhdCBtb2RlcmF0ZWx5IGNvdmFyeS4KCmBgYHtyfQojICMgbXlzZWVkIDwtIHJ1bmlmKDEsIDAsIDEwMDApCiMgbXlzZWVkIDwtIDQwMS42MzY4CiMgCiMgZGYgPC0gZGF0YV9mcmFtZShncm91cCA9IHJlcChjKCJhIiwgImIiKSwgZWFjaCA9IDEwKSkKIyBub19kaXNjciA8LSBkZiAlPiUKIyAgIHNpbV9jb3Zhcig1LCB2YXIgPSAxLCBjb3YgPSAwLCBuYW1lID0gInVuY29yciIsIHNlZWQgPSBteXNlZWQpICU+JSAKIyAgIHNpbV9jb3ZhcigxMCwgdmFyID0gMSwgY292ID0gMC41LCBuYW1lID0gImNvcnJBIiwgc2VlZCA9IG15c2VlZCArIDEpICU+JSAKIyAgIHNpbV9jb3ZhcigxMCwgdmFyID0gMSwgY292ID0gMC41LCBuYW1lID0gImNvcnJCIiwgc2VlZCA9IG15c2VlZCArIDIpCmBgYAoKVGhpcyBoZWF0bWFwIHNob3dzIHRoZSBjb3JyZWxhdGlvbiBzdHJ1Y3R1cmUuCgpgYGB7cn0KIyBub19kaXNjciAlPiUgCm51bGwgJT4lICAgCiAgc2VsZWN0KC1ncm91cCkgJT4lIAogIGNvcigpICU+JSAKICBpaGVhdG1hcChyb3dfbGFiZWxzID0gVFJVRSwgY29sX2xhYmVscyA9IFRSVUUsIG5hbWUgPSAiY29yIikKYGBgCmBgYHtyfQpudWxsLmNvciA8LQogICMgbm9fZGlzY3IgJT4lIAogIG51bGwgJT4lIAogIHNlbGVjdCgtZ3JvdXApICU+JSAKICBjb3IoKSAlPiUKICBhc19kYXRhX2ZyYW1lKHJvd25hbWVzID0gInJvdyIpICU+JSAKICBnYXRoZXIoLXJvdywga2V5ID0gY29sLCB2YWx1ZSA9IGNvcikKCiMgY29yMSA8LQpudWxsLmNvci5wIDwtCiAgZ2dwbG90KG51bGwuY29yLCBhZXMoeCA9IGNvbCwgeSA9IHJvdywgZmlsbCA9IGNvcikpICsKICBnZW9tX3RpbGUoKSArCiAgc2NhbGVfZmlsbF9ncmFkaWVudDIoIkNvcnJlbGF0aW9uIENvZWZpY2llbnQiKSArCiAgbGFicyh4ID0gTlVMTCwgeSA9IE5VTEwpICsKICB0aGVtZV9taW5pbWFsKCkgKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gOTApLCBsZWdlbmQucG9zaXRpb24gPSAidG9wIikKYGBgCgpUaGVuIEkgcGVyZm9ybSBQQ0EgYW5kIFBMUy1EQSBvbiB0aGlzIGRhdGFzZXQsIGZvcmNpbmcgYm90aCBtb2RlbHMgdG8gaGF2ZSB0d28gYXhlcyAoZm9yIHBsb3R0aW5nKS4gIFRoZSBQQ0Egc2NvcmUgcGxvdCBzaG93cyBubyBzZXBhcmF0aW9uIGFuZCB0aGUgZmlyc3QgdHdvIGF4ZXMgZXhwbGFpbiBtb3JlIHRoYW4gaGFsZiBvZiB0aGUgdmFyaWF0aW9uIGluIHRoZSBkYXRhLiAgVGhlIFBMUy1EQSBzY29yZSBwbG90IHNob3dzIHNsaWdodCBzZXBhcmF0aW9uIG9mIHRoZSBncm91cHMgKGV2ZW4gdGhvdWdoIHRoZXJlIGlzIG5vbmUpLiAgSXQncyBhIHByZXR0eSB0ZXJyaWJsZSBtb2RlbCB3aXRoIGEgbmVnYXRpdmUgUV4yXiB2YWx1ZSBhbmQgYSBsb3cgUl4yXiB2YWx1ZS4KCmBgYHtyIGluY2x1ZGU9RkFMU0V9Cm51bGwucGNhIDwtIG9wbHMoc2VsZWN0KG51bGwsIC1ncm91cCksIHBsb3QgPSBGQUxTRSkKbnVsbC5wbHMgPC0gb3BscyhzZWxlY3QobnVsbCwgLWdyb3VwKSwgbnVsbCRncm91cCwgcGxvdCA9IEZBTFNFLCBwcmVkSSA9IDIsIHBlcm1JID0gNTAwKQpgYGAKCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9Cm51bGwucGNhLnAgPC0gcGxvdF9wY2EobnVsbC5wY2EsIG51bGwkZ3JvdXAsIGFubm90YXRlID0gInN1YnRpdGxlIikKbnVsbC5wbHMucCA8LSBwbG90X3Bsc2RhKG51bGwucGxzLCBhbm5vdGF0ZSA9ICJzdWJ0aXRsZSIpCnBsb3RfZ3JpZChudWxsLmNvci5wLAogICAgICAgICAgbnVsbC5wY2EucCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJOVUxMIiksCiAgICAgICAgICBudWxsLnBscy5wICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIk5VTEwiKSwgbnJvdyA9IDEsIG5jb2wgPSAzKQpgYGAKCiMgTmVlZGxlIGluIGEgaGF5c3RhY2sKCktlZXBpbmcgdGhlIHRvdGFsIG51bWJlciBvZiBvYnNlcnZhdGlvbnMgYW5kIHZhcmlhYmxlcyB0aGUgc2FtZSwgYW5kIHVzaW5nIHRoZSBzYW1lIHJhbmRvbSBzZWVkLCBJIGNyZWF0ZSBhIHNlY29uZCBkYXRhIHNldCB3aXRoIDUgdmFyaWFibGVzIHRoYXQgZG8gbm90IGNvdmFyeSB3aXRoaW4gZ3JvdXBzLCBidXQgZGlmZmVyIGluIHRoZWlyIG1lYW5zIGJldHdlZW4gZ3JvdXBzLgoKYGBge3J9CiMgZGlzY3IgPC0gZGYgJT4lIAojICAgc2ltX2NvdmFyKDEwLCB2YXIgPSAxLCBjb3YgPSAwLjU1LCBuYW1lID0gImNvcnJBIiwgc2VlZCA9IG15c2VlZCArIDEpICU+JSAKIyAgIHNpbV9jb3ZhcigxMCwgdmFyID0gMSwgY292ID0gMC41NSwgbmFtZSA9ICJjb3JyQiIsIHNlZWQgPSBteXNlZWQgKyAyKSAlPiUgCiMgICBzaW1fZGlzY3IoNSwgdmFyID0gMSwgY292ID0gMCwgZ3JvdXBfbWVhbnMgPSBjKC0xLCAxKSwgbmFtZSA9ICJkaXNjciIsIHNlZWQgPSBteXNlZWQgKyAzKQpgYGAKClRoaXMgc2hvd3MgdGhlIGNvcnJlbGF0aW9uIHN0cnVjdHVyZSBvZiB0aGUgZGF0YS4gIEJlY2F1c2UgdGhlIGRpc2NyaW1pbmF0aW5nIHZhcmlhYmxlcyBoYXZlIGRpZmZlcmVudCBtZWFucyBpbiB0aGUgdHdvIGdyb3VwcywgdGhleSBhcmUgY29ycmVsYXRlZCBhYm91dCBhcyBzdHJvbmdseSBhcyB0aGUgY292YXJ5aW5nIHZhcmlhYmxlcy4KCmBgYHtyfQpuZWVkbGUgJT4lIAojIGRpc2NyICU+JSAKICBzZWxlY3QoLWdyb3VwKSAlPiUgCiAgY29yKCkgJT4lIAogIGloZWF0bWFwKHJvd19sYWJlbHMgPSBUUlVFLCBjb2xfbGFiZWxzID0gVFJVRSwgbmFtZSA9ICJjb3IiKQpgYGAKYGBge3J9Cm5lZWRsZS5jb3IgPC0KICBuZWVkbGUgJT4lIAogICMgZGlzY3IgJT4lIAogIHNlbGVjdCgtZ3JvdXApICU+JSAKICBjb3IoKSAlPiUKICBhc19kYXRhX2ZyYW1lKHJvd25hbWVzID0gInJvdyIpICU+JSAgCiAgZ2F0aGVyKC1yb3csIGtleSA9IGNvbCwgdmFsdWUgPSBjb3IpCgpuZWVkbGUuY29yLnAgPC0KICBnZ3Bsb3QobmVlZGxlLmNvciwgYWVzKHggPSBjb2wsIHkgPSByb3csIGZpbGwgPSBjb3IpKSArCiAgZ2VvbV90aWxlKCkgKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKCJDb3JyZWxhdGlvbiBDb2VmaWNpZW50IikgKwogIGxhYnMoeCA9IE5VTEwsIHkgPSBOVUxMKSArCiAgdGhlbWVfbWluaW1hbCgpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDkwKSwgbGVnZW5kLnBvc2l0aW9uID0gInRvcCIpCiMgbmVlZGxlLmNvci5wCmBgYApUaGUgUENBIHNjb3JlIHBsb3Qgc3RpbGwgc2hvd3Mgbm8gcmVhbCBzZXBhcmF0aW9uIGJlY2F1c2UgdGhlIGRpc2NyaW1pbmF0aW5nIHZhcmlhYmxlcyBkbyBub3QgY29udHJpYnV0ZSBtdWNoIHRvIHRoZSB0b3RhbCBjb3ZhcmlhdGlvbiBpbiB0aGUgZGF0YS4gIFRoZSBQTFMtREEgcGxvdCBzaG93cyB2ZXJ5IHN0cm9uZyBzZXBhcmF0aW9uLCBkZXNwaXRlIHRoZSBkaWZmZXJlbmNlcyBiZXR3ZWVuIHRoZSBncm91cHMgYmVpbmcgb25seSBkdWUgdG8gNSBvdXQgb2YgMjUgdmFyaWFibGVzLiAgVGhlIFBMUy1EQSBtb2RlbCBwZXJmb3JtcyB3ZWxsLCB3aXRoIGhpZ2ggUl4yXiBhbmQgUV4yXiB2YWx1ZXMKCmBgYHtyIGluY2x1ZGU9RkFMU0V9Cm5lZWRsZS5wY2EgPC0gb3BscyhzZWxlY3QobmVlZGxlLCAtZ3JvdXApLCBwbG90ID0gRkFMU0UpCm5lZWRsZS5wbHMgPC0gb3BscyhzZWxlY3QobmVlZGxlLCAtZ3JvdXApLCBuZWVkbGUkZ3JvdXAsIHBsb3QgPSBGQUxTRSwgcHJlZEkgPSAyLCBwZXJtSSA9IDUwMCkKYGBgCgpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpuZWVkbGUucGNhLnAgPC0gcGxvdF9wY2EobmVlZGxlLnBjYSwgbmVlZGxlJGdyb3VwLCBhbm5vdGF0ZSA9ICJzdWJ0aXRsZSIpCm5lZWRsZS5wbHMucCA8LSBwbG90X3Bsc2RhKG5lZWRsZS5wbHMsIGFubm90YXRlID0gInN1YnRpdGxlIikKcGxvdF9ncmlkKG5lZWRsZS5jb3IucCwKICAgICAgICAgIG5lZWRsZS5wY2EucCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJOVUxMIiksCiAgICAgICAgICBuZWVkbGUucGxzLnAgKyB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiTlVMTCIpLAogICAgICAgICAgbnJvdyA9IDEsIG5jb2wgPSAzKQpgYGAKCiMgQ29udHJvbApgYGB7cn0KY29udHJvbC5jb3IgPC0KICBjb250cm9sICU+JSAKICBzZWxlY3QoLWdyb3VwKSAlPiUgCiAgY29yKCkgJT4lCiAgYXNfZGF0YV9mcmFtZShyb3duYW1lcyA9ICJyb3ciKSAlPiUgCiAgZ2F0aGVyKC1yb3csIGtleSA9IGNvbCwgdmFsdWUgPSBjb3IpCgpjb250cm9sLmNvci5wIDwtCiAgZ2dwbG90KGNvbnRyb2wuY29yLCBhZXMoeCA9IGNvbCwgeSA9IHJvdywgZmlsbCA9IGNvcikpICsKICBnZW9tX3RpbGUoKSArCiAgc2NhbGVfZmlsbF9ncmFkaWVudDIoIkNvcnJlbGF0aW9uIENvZWZpY2llbnQiKSArCiAgbGFicyh4ID0gTlVMTCwgeSA9IE5VTEwpICsKICB0aGVtZV9taW5pbWFsKCkgKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gOTApLCBsZWdlbmQucG9zaXRpb24gPSAidG9wIikKYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQpjb250cm9sLnBjYSA8LSBvcGxzKHNlbGVjdChjb250cm9sLCAtZ3JvdXApLCBwbG90ID0gRkFMU0UpCmNvbnRyb2wucGxzIDwtIG9wbHMoc2VsZWN0KGNvbnRyb2wsIC1ncm91cCksIGNvbnRyb2wkZ3JvdXAsIHBsb3QgPSBGQUxTRSwgcHJlZEkgPSAyLCBwZXJtSSA9IDUwMCkKYGBgCgoKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KY29udHJvbC5wY2EucCA8LSBwbG90X3BjYShjb250cm9sLnBjYSwgY29udHJvbCRncm91cCwgYW5ub3RhdGUgPSAic3VidGl0bGUiKQpjb250cm9sLnBscy5wIDwtIHBsb3RfcGxzZGEoY29udHJvbC5wbHMsIGFubm90YXRlID0gInN1YnRpdGxlIikKcGxvdF9ncmlkKGNvbnRyb2wuY29yLnAsCiAgICAgICAgICBjb250cm9sLnBjYS5wICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIk5VTEwiKSwKICAgICAgICAgIGNvbnRyb2wucGxzLnAgKyB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAiTlVMTCIpLAogICAgICAgICAgbnJvdyA9IDEsIG5jb2wgPSAzKQpgYGAKCiMgU2F2ZSBwbG90cyBhbmQgdGFibGVzCgpgYGB7cn0KZmlnMV9uZXcgPC0KICBwbG90X2dyaWQobnVsbC5jb3IucCwKICAgICAgICAgICAgbnVsbC5wY2EucCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikgKyBsYWJzKHRpdGxlID0gIiIpLAogICAgICAgICAgICBudWxsLnBscy5wICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiKSArIGxhYnModGl0bGUgPSAiIiksCiAgICAgICAgICAgIGNvbnRyb2wuY29yLnAsCiAgICAgICAgICAgIGNvbnRyb2wucGNhLnAgKyB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAibm9uZSIpICsgbGFicyh0aXRsZSA9ICIiKSwKICAgICAgICAgICAgY29udHJvbC5wbHMucCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikgKyBsYWJzKHRpdGxlID0gIiIpLAogICAgICAgICAgICBuZWVkbGUuY29yLnAsCiAgICAgICAgICAgIG5lZWRsZS5wY2EucCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikgKyBsYWJzKHRpdGxlID0gIiIpLAogICAgICAgICAgICBuZWVkbGUucGxzLnAgKyB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAibm9uZSIpICsgbGFicyh0aXRsZSA9ICIiKSwKICAgICAgICAgICAgbmNvbCA9IDMsIG5yb3cgPSAzLCBsYWJlbHMgPSAiQVVUTyIpCgpzYXZlX3Bsb3QoaGVyZSgiZmlncyIsICJmaWcxX25ldy5wbmciKSwgZmlnMV9uZXcsIG5jb2wgPSAzLCBucm93ID0gMykKYGBgCgoKCgoKIyBvbGQsIG5vdCBydW4KClZJUCBzY29yZXMgZnJvbSB0aGUgUExTLURBIG1vZGVsIGlkZW50aWZ5IGFsbCA1IGRpc2NyaW1pbmF0aW5nIHZhcmlhYmxlcyBhcyBiZWluZyBpbXBvcnRhbnQgZm9yIHNlcGFyYXRpbmcgdGhlIGdyb3VwcyAoVklQID4gMSkuICBVc2luZyB0aGUgVklQID4gMSBjdXRvZmYsIGl0IGFsc28gc3B1cnJpb3VzbHkgaWRlbnRpZmllcyB0d28gb2YgdGhlIG90aGVyIHZhcmlhYmxlcyBhcyBiZWluZyBpbXBvcnRhbnQuICBEaXNjcmltaW5hdGluZyB2YXJpYWJsZXMgYXJlIG5vdCBzdHJvbmdseSBjb3JyZWxhdGVkIHdpdGggdGhlIGZpcnN0IHR3byBheGVzIG9mIHRoZSBQQ0EuCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpWSVBzIDwtIGdldF9WSVAocGxzMikgJT4lCiAgYXJyYW5nZShkZXNjKFZJUCkpCgpwY2EubG9hZCA8LSBnZXRfbG9hZGluZ3MocGNhMikgJT4lCiAgc2VsZWN0KFZhcmlhYmxlLCBwMSwgcDIpICU+JQogIGFycmFuZ2UoZGVzYyhhYnMocDEpKSkKYGBgCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQp0YWJsZTEgPC0gZnVsbF9qb2luKFZJUHMsIHBjYS5sb2FkKSAlPiUKICBhcnJhbmdlKFZhcmlhYmxlKSAlPiUgCiAgcmVuYW1lKGBQTFMtREEgVklQIHNjb3JlYCA9IFZJUCwgYFBDMSBsb2FkaW5nYCA9IHAxLCBgUEMyIGxvYWRpbmdgID0gcDIpICU+JSAKICBtdXRhdGVfaWYoaXMuZG91YmxlLCB+cm91bmQoLiw0KSkKdGFibGUxCndyaXRlX2V4Y2VsX2Nzdih0YWJsZTEsICJmaWdzL3RhYmxlMS5jc3YiKQpgYGAKCiMgUmVkIEhlcnJpbmcKTGFyZ2UgdmFyaWF0aW9uIHdpdGggd2VhayBkaXNjcmltaW5hdGlvbiBwbHVzIHNtYWxsIHZhcmlhdGlvbiB3aXRoIGxhcmdlIGRpc2NyaW1pbmF0aW9uCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpoZXJyaW5nIDwtIGRmICU+JSAKICBzaW1fZGlzY3IocCA9IDEwLCB2YXIgPSAxLCBjb3YgPSAwLjUsIGdyb3VwX21lYW5zID0gYygtMC41LCAwLjUpLCBuYW1lID0gImhpZ2h2YXIiKSAlPiUgCiAgc2ltX2Rpc2NyKHAgPSAxMCwgdmFyID0gMSwgY292ID0gMCwgZ3JvdXBfbWVhbnMgPSBjKC0xLCAxKSwgbmFtZSA9ICJsb3d2YXIiKSAlPiUgCiAgc2ltX2NvdmFyKHAgPSA1LCB2YXIgPSAxLCBjb3YgPSAwLCBuYW1lID0gIm5vaXNlIikKYGBgCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpoZXJyaW5nICU+JSAKICBzZWxlY3QoLWdyb3VwKSAlPiUgCiAgY29yKCkgJT4lIAogIGloZWF0bWFwKHJvd19sYWJlbHMgPSBUUlVFLCBjb2xfbGFiZWxzID0gVFJVRSwgbmFtZSA9ICJjb3IiKQpgYGAKYGBge3IgZXZhbD1GQUxTRSwgaW5jbHVkZT1GQUxTRX0KaGVycmluZy5jb3IgPC0gaGVycmluZyAlPiUgCiAgc2VsZWN0KC1ncm91cCkgJT4lIAogIGNvcigpICU+JQogIGFzLmRhdGEuZnJhbWUoKSAlPiUgCiAgcm93bmFtZXNfdG9fY29sdW1uKHZhciA9ICJyb3ciKSAlPiUgCiAgZ2F0aGVyKC1yb3csIGtleSA9IGNvbCwgdmFsdWUgPSBjb3IpCgpjb3IzIDwtIGdncGxvdChoZXJyaW5nLmNvciwgYWVzKHggPSBjb2wsIHkgPSByb3csIGZpbGwgPSBjb3IpKSArCiAgZ2VvbV90aWxlKCkgKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKCJDb3JyZWxhdGlvbiBDb2VmaWNpZW50IikgKwogIGxhYnMoeCA9IE5VTEwsIHkgPSBOVUxMKSArCiAgdGhlbWVfbWluaW1hbCgpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDkwKSwgbGVnZW5kLnBvc2l0aW9uID0gInRvcCIpCmBgYAoKCmBgYHtyIGV2YWw9RkFMU0UsIGluY2x1ZGU9RkFMU0V9CnBjYTMgPC0gb3BscyhzZWxlY3QoaGVycmluZywgLWdyb3VwKSwgcGxvdCA9IEZBTFNFKQpwbHMzIDwtIG9wbHMoc2VsZWN0KGhlcnJpbmcsIC1ncm91cCksIG5vX2Rpc2NyJGdyb3VwLCBwbG90ID0gRkFMU0UsIHByZWRJID0gMiwgcGVybUkgPSA1MDApCmBgYApgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpwY2EzLnBsb3QgPC0gcGxvdF9wY2EocGNhMywgaGVycmluZyRncm91cCwgYW5ub3RhdGUgPSAic3VidGl0bGUiKQpwbHMzLnBsb3QgPC0gcGxvdF9wbHNkYShwbHMzLCBhbm5vdGF0ZSA9ICJzdWJ0aXRsZSIpCnBsb3RfZ3JpZChjb3IzLAogICAgICAgICAgcGNhMy5wbG90ICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIk5VTEwiKSwKICAgICAgICAgIHBsczMucGxvdCArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJOVUxMIiksIG5yb3cgPSAxLCBuY29sID0gMykKYGBgCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpWSVBzMyA8LSBnZXRfVklQKHBsczMpICU+JQogIGFycmFuZ2UoZGVzYyhWSVApKQoKcGNhMy5sb2FkIDwtIGdldF9sb2FkaW5ncyhwY2EzKSAlPiUKICBzZWxlY3QoVmFyaWFibGUsIHAxLCBwMikgJT4lCiAgYXJyYW5nZShkZXNjKGFicyhwMSkpKQoKdGFibGUzIDwtIGZ1bGxfam9pbihWSVBzMywgcGNhMy5sb2FkKSAlPiUKICBhcnJhbmdlKFZhcmlhYmxlKSAlPiUgCiAgcmVuYW1lKGBQTFMtREEgVklQIHNjb3JlYCA9IFZJUCwgYFBDMSBsb2FkaW5nYCA9IHAxLCBgUEMyIGxvYWRpbmdgID0gcDIpICU+JSAKICBtdXRhdGVfaWYoaXMuZG91YmxlLCB+cm91bmQoLiw0KSkKdGFibGUzICU+JSBhcnJhbmdlKGRlc2MoYFBMUy1EQSBWSVAgc2NvcmVgKSkKdGFibGUzICU+JSBhcnJhbmdlKGRlc2MoYFBDMSBsb2FkaW5nYCkpCiMgd3JpdGVfZXhjZWxfY3N2KHRhYmxlMSwgImZpZ3MvdGFibGUxLmNzdiIpCgpgYGAKCgoKCgoK